Dependencies

library(ggplot2) #core graphics package
library(ggridges) # ggarrange
library(ggpubr)
library(ggdist) 
library(ggtext)

library(tidyverse) # data manipulation

library(rstan) #detectCores
library(brms) # core pack
library(parallel) #detectCores
library(tidybayes) 


REBUILD_FLAG <<- FALSE

Load data

metas <- read.csv("../data/metas.csv", stringsAsFactors = F, header = T)
trials_all <- read.csv("../data/trials.csv", stringsAsFactors = F, header = T)
seper <- " o "

trials_all <- trials_all %>%
          filter(TrialStatus == "main") %>% #drop practice trial
          mutate(Error = TrialAnswer - TrialIntAccuracy,
                 ErrorMagnitude = abs(Error),
                 ResponseTime = (TrialVisualizationEnd - TrialVisualizationStart) / 1000,
                 LogResponseTime = log(ResponseTime)) %>% # log transform
      filter(!(ParticipantID == "P7" & SessionIndex == 0 & TrialStatus == "main" & TrialIndex == 16)) %>%  # filter mistakes
      filter(!(ParticipantID == "P9" & SessionIndex == 2 & TrialStatus == "main" & TrialIndex == 17)) %>% 
      filter(!(ParticipantID == "P11" & SessionIndex == 0 & TrialStatus == "main"  & TrialIndex == 12)) %>% 
      filter(!(ParticipantID == "P20" & SessionIndex == 3 & TrialStatus == "main" & TrialIndex == 5)) %>% 
      filter(!(ParticipantID == "P24" & SessionIndex == 0 & TrialStatus == "main"  & TrialIndex == 26)) %>% 
      filter(ErrorMagnitude < 51) %>% 
      mutate(  # manual recode levels using orthogonal contrast coding       
             AllFactors = paste(FactorStereo, FactorMotion, FactorPerspective, FactorShading, Dim, sep = seper),
             Stereo = recode(FactorStereo, yes = 0.5, no = -0.5),
             Motion = recode(FactorMotion, yes = 0.5, no = -0.5),
             Perspective = recode(FactorPerspective, yes = 0.5, no = -0.5),
             Shading = recode(FactorShading, `shading-1` = 0, `shading-2` = 0.5, `flat`= -0.5, `no` = -0.5),
             Dataset = recode(TrialDataset, "cifar10" = 0.5, "babi-q1" = -0.5),
             Dimension = recode(Dim, `2` = -0.5, `3` = 0.5)
             ) 


trials <- trials_all %>%
  select(ParticipantID, ErrorMagnitude, ResponseTime, Stereo, Motion, Perspective, Shading, Dimension, Dataset)


vtrials <- trials_all %>%
  select(ParticipantID, ErrorMagnitude, ResponseTime, FactorStereo, FactorMotion, FactorPerspective, FactorShading, Dim, TrialDataset, AllFactors)

Raw data

We first plot all raw data to get a sense of how data distribute. Both error magnitude and response time datum seem not to follow a normal distribution, and they are both non-negative. A skewed normal, lognormal, and gamma distribution could be reasonable. Note that error magnitude contains 0, and response time has a upper bound 60s (e.g., 60.1s, 60.2s due to running time). We will use gamma distribution for both measures. To formally compare which

g_raw_error <- vtrials %>%
         ggplot(mapping = aes(x = ErrorMagnitude)) +
         facet_wrap(AllFactors ~ .,ncol = 3, scales = "free") +
         coord_cartesian(xlim = c(0, 50), ylim = c(0, 30)) + 
         geom_histogram(binwidth = 2, fill = "lightblue", color = "white") +
         ggtitle("raw data - error magnitude (%)") + 
         xlab ("error magnitude ") + ylab("count")
   
g_raw_error

ggsave(g_raw_error, filename = "pic/g_raw_error.pdf", width = 10, height = 10, units = "in")

g_raw_time <- vtrials %>%
         ggplot(mapping = aes(x = ResponseTime)) +
         facet_wrap(AllFactors ~ .,ncol = 3, scales = "free") +
         coord_cartesian(xlim = c(0, 62), ylim = c(0, 30)) + 
         geom_histogram(binwidth = 2, fill = "lightgray", color = "white") +
         ggtitle("raw data - response time  (s)") + 
         xlab ("response time ") + ylab("count")
   
g_raw_time

ggsave(g_raw_time, filename = "pic/g_raw_time.pdf", width = 10, height = 10, units = "in")

helper function for model diagnostics

model_diagnostics <- function(m, name = "model diagnostics"){
  print(name)
  
  print(summary(m))

  # plot(m)
  
  plot(pp_check(m, type = "dens_overlay", nsamples = 100))
  plot(pp_check(m, type = "stat_2d", nsamples = 100))
  plot(pp_check(m, type = "error_binned", nsamples = 100))
  
  print(loo(m))
  print(waic(m))
}

# Valid types are:
# 'bars', 'bars_grouped', 'boxplot', 'data', 'dens', 'dens_overlay', 'ecdf_overlay', 'error_binned', 'error_hist', 'error_hist_grouped', 'error_scatter', 'error_scatter_avg', 'error_scatter_avg_vs_x', 'freqpoly', 'freqpoly_grouped', 'hist', 'intervals', 'intervals_data', 'intervals_grouped', 'loo_intervals', 'loo_pit', 'loo_pit_overlay', 'loo_pit_qq', 'loo_ribbon', 'ribbon', 'ribbon_data', 'ribbon_grouped', 'rootogram', 'scatter', 'scatter_avg', 'scatter_avg_grouped', 'stat', 'stat_2d', 'stat_freqpoly_grouped', 'stat_grouped', 'violin_grouped'
if(REBUILD_FLAG){

  m_error_hurdle_gamma <- bf(ErrorMagnitude ~  
                           0 + Intercept + 
                           Stereo*Motion*Perspective*Shading*Dataset +  
                           Stereo*Dataset*Dimension + 
                           (1 + Dimension | ParticipantID), 
                           family = hurdle_gamma(link = "identity")
                        ) 
  
  get_prior(m_error_hurdle_gamma, data = trials)

    
  m_brms_error_hurdle_gamma <<- brm(m_error_hurdle_gamma, 
                           data = trials, 
                           family = hurdle_gamma(link = "log"), 
                           prior = c(
                              prior(normal(0, 2.5), class = "b"),
                              prior(normal(0, 25),  class = "b", coef = "Intercept"), 
                              prior(gamma(1, 10), class = "shape"),
                              prior(student_t(3, 0, 5), class = "sd")
                            ), 
                            chains = N_CHAINS, iter = N_ITER, warmup = N_WARMUP, thin = N_THIN, cores = N_CORES,
                            control = list(adapt_delta = DELTA, max_treedepth = TREE_DEPTH))
  
  saveRDS(file = "rds/m_brms_error_hurdle_gamma.rds", m_brms_error_hurdle_gamma)
  
}else{
  m_brms_error_hurdle_gamma <<- readRDS("rds/m_brms_error_hurdle_gamma.rds")
}

model_diagnostics(m_brms_error_hurdle_gamma)
## [1] "model diagnostics"
##  Family: hurdle_gamma 
##   Links: mu = log; shape = identity; hu = identity 
## Formula: ErrorMagnitude ~ 0 + Intercept + Stereo * Motion * Perspective * Shading * Dataset + Stereo * Dataset * Dimension + (1 + Dimension | ParticipantID) 
##    Data: trials (Number of observations: 3320) 
## Samples: 2 chains, each with iter = 4000; warmup = 2000; thin = 1;
##          total post-warmup samples = 4000
## 
## Group-Level Effects: 
## ~ParticipantID (Number of levels: 32) 
##                          Estimate Est.Error l-95% CI u-95% CI Rhat Bulk_ESS
## sd(Intercept)                0.20      0.04     0.13     0.29 1.00     1542
## sd(Dimension)                0.13      0.08     0.01     0.32 1.00      548
## cor(Intercept,Dimension)     0.18      0.45    -0.71     0.96 1.00     1945
##                          Tail_ESS
## sd(Intercept)                2254
## sd(Dimension)                1055
## cor(Intercept,Dimension)     1524
## 
## Population-Level Effects: 
##                                           Estimate Est.Error l-95% CI u-95% CI
## Intercept                                     1.96      0.05     1.86     2.06
## Stereo                                       -0.03      0.07    -0.17     0.10
## Motion                                       -0.03      0.03    -0.09     0.03
## Perspective                                   0.01      0.03    -0.05     0.07
## Shading                                      -0.04      0.04    -0.11     0.03
## Dataset                                       0.06      0.07    -0.08     0.20
## Dimension                                     0.05      0.07    -0.09     0.20
## Stereo:Motion                                 0.10      0.06    -0.02     0.22
## Stereo:Perspective                           -0.03      0.06    -0.15     0.09
## Motion:Perspective                           -0.04      0.06    -0.16     0.07
## Stereo:Shading                               -0.00      0.07    -0.15     0.14
## Motion:Shading                                0.06      0.07    -0.08     0.21
## Perspective:Shading                           0.03      0.07    -0.11     0.17
## Stereo:Dataset                                0.13      0.14    -0.15     0.42
## Motion:Dataset                               -0.11      0.08    -0.27     0.04
## Perspective:Dataset                           0.07      0.06    -0.05     0.19
## Shading:Dataset                               0.00      0.07    -0.13     0.14
## Stereo:Dimension                              0.13      0.14    -0.14     0.40
## Dataset:Dimension                             0.11      0.14    -0.16     0.39
## Stereo:Motion:Perspective                     0.09      0.12    -0.15     0.32
## Stereo:Motion:Shading                         0.15      0.14    -0.13     0.42
## Stereo:Perspective:Shading                    0.23      0.14    -0.06     0.51
## Motion:Perspective:Shading                    0.03      0.15    -0.26     0.31
## Stereo:Motion:Dataset                         0.12      0.16    -0.19     0.44
## Stereo:Perspective:Dataset                    0.15      0.12    -0.07     0.39
## Motion:Perspective:Dataset                    0.02      0.12    -0.21     0.25
## Stereo:Shading:Dataset                        0.08      0.15    -0.21     0.37
## Motion:Shading:Dataset                        0.30      0.14     0.02     0.58
## Perspective:Shading:Dataset                  -0.08      0.14    -0.36     0.21
## Stereo:Dataset:Dimension                     -0.21      0.29    -0.77     0.35
## Stereo:Motion:Perspective:Shading             0.11      0.28    -0.43     0.66
## Stereo:Motion:Perspective:Dataset            -0.08      0.24    -0.56     0.38
## Stereo:Motion:Shading:Dataset                 0.28      0.29    -0.28     0.83
## Stereo:Perspective:Shading:Dataset           -0.01      0.29    -0.58     0.56
## Motion:Perspective:Shading:Dataset            0.34      0.29    -0.23     0.91
## Stereo:Motion:Perspective:Shading:Dataset    -0.39      0.57    -1.52     0.74
##                                           Rhat Bulk_ESS Tail_ESS
## Intercept                                 1.00     1653     2140
## Stereo                                    1.00     2732     2683
## Motion                                    1.00     5316     3176
## Perspective                               1.00     5212     3062
## Shading                                   1.00     4335     2956
## Dataset                                   1.00     2731     2413
## Dimension                                 1.00     3814     3064
## Stereo:Motion                             1.00     5224     2852
## Stereo:Perspective                        1.00     4497     2730
## Motion:Perspective                        1.00     4957     3034
## Stereo:Shading                            1.00     5547     3077
## Motion:Shading                            1.00     5531     3101
## Perspective:Shading                       1.00     4795     2890
## Stereo:Dataset                            1.00     3066     2820
## Motion:Dataset                            1.00     3970     2321
## Perspective:Dataset                       1.00     5108     2817
## Shading:Dataset                           1.00     4535     3253
## Stereo:Dimension                          1.00     2558     2647
## Dataset:Dimension                         1.00     2749     2875
## Stereo:Motion:Perspective                 1.00     4908     3258
## Stereo:Motion:Shading                     1.00     4827     2927
## Stereo:Perspective:Shading                1.00     5124     2832
## Motion:Perspective:Shading                1.00     5449     3177
## Stereo:Motion:Dataset                     1.00     3704     3034
## Stereo:Perspective:Dataset                1.00     5342     2920
## Motion:Perspective:Dataset                1.00     5247     3347
## Stereo:Shading:Dataset                    1.00     5093     3131
## Motion:Shading:Dataset                    1.00     4711     3042
## Perspective:Shading:Dataset               1.00     4890     2809
## Stereo:Dataset:Dimension                  1.00     3099     2736
## Stereo:Motion:Perspective:Shading         1.00     5169     3126
## Stereo:Motion:Perspective:Dataset         1.00     5398     3292
## Stereo:Motion:Shading:Dataset             1.00     5028     2555
## Stereo:Perspective:Shading:Dataset        1.00     6034     3406
## Motion:Perspective:Shading:Dataset        1.00     4366     3316
## Stereo:Motion:Perspective:Shading:Dataset 1.00     4858     3179
## 
## Family Specific Parameters: 
##       Estimate Est.Error l-95% CI u-95% CI Rhat Bulk_ESS Tail_ESS
## shape     1.58      0.04     1.50     1.65 1.00     5576     2806
## hu        0.07      0.00     0.06     0.08 1.00     5625     2948
## 
## Samples were drawn using sampling(NUTS). For each parameter, Bulk_ESS
## and Tail_ESS are effective sample size measures, and Rhat is the potential
## scale reduction factor on split chains (at convergence, Rhat = 1).

## 
## Computed from 4000 by 3320 log-likelihood matrix
## 
##          Estimate    SE
## elpd_loo  -9939.7  50.9
## p_loo        73.2   3.1
## looic     19879.4 101.8
## ------
## Monte Carlo SE of elpd_loo is 0.2.
## 
## Pareto k diagnostic values:
##                          Count Pct.    Min. n_eff
## (-Inf, 0.5]   (good)     3317  99.9%   922       
##  (0.5, 0.7]   (ok)          3   0.1%   804       
##    (0.7, 1]   (bad)         0   0.0%   <NA>      
##    (1, Inf)   (very bad)    0   0.0%   <NA>      
## 
## All Pareto k estimates are ok (k < 0.7).
## See help('pareto-k-diagnostic') for details.
## 
## Computed from 4000 by 3320 log-likelihood matrix
## 
##           Estimate    SE
## elpd_waic  -9939.2  50.9
## p_waic        72.8   3.0
## waic       19878.5 101.7
## 
## 13 (0.4%) p_waic estimates greater than 0.4. We recommend trying loo instead.
if(REBUILD_FLAG == TRUE){
  
  trials <- trials %>%
      mutate(cen = ifelse(ResponseTime > 60, "left", "none"))

  m_time_gamma_motion <- bf(ResponseTime | cens(cen) ~  
                      0 + Intercept +
                      Stereo*Motion*Perspective*Shading*Dataset +  
                      Stereo*Dataset*Dimension + 
                      (1 + Motion | ParticipantID),
                      shape ~ 0 + Intercept + Motion,
                      family = Gamma(link="log")
                     ) 
    
  priors_time_gamma_motion <- c(
                set_prior("normal(0, 2.5)", class = "b"),
                set_prior("normal(0, 30)", class = "b", coef = "Intercept"), 
                #prior(gamma(1, 20), class = "shape"),
                set_prior("student_t(3, 0, 10)", class = "sd")
                ) 
    
  m_brms_time_gamma_motion <<- brm(m_time_gamma_motion, 
                              data = trials, family = Gamma(link="log"),  prior = priors_time_gamma_motion, 
                              chains = N_CHAINS, iter = N_ITER, warmup = N_WARMUP, thin = N_THIN, cores = N_CORES,
                              control = list(adapt_delta = DELTA, max_treedepth = TREE_DEPTH))
  
  saveRDS(file = "rds/m_brms_time_gamma_motion.rds", m_brms_time_gamma_motion)
  
}else{
  
  m_brms_time_gamma_motion <<- readRDS("rds/m_brms_time_gamma_motion.rds")
}

model_diagnostics(m_brms_time_gamma_motion)
## [1] "model diagnostics"
##  Family: gamma 
##   Links: mu = log; shape = log 
## Formula: ResponseTime | cens(cen) ~ 0 + Intercept + Stereo * Motion * Perspective * Shading * Dataset + Stereo * Dataset * Dimension + (1 + Motion | ParticipantID) 
##          shape ~ 0 + Intercept + Motion
##    Data: trials (Number of observations: 3320) 
## Samples: 2 chains, each with iter = 4000; warmup = 2000; thin = 1;
##          total post-warmup samples = 4000
## 
## Group-Level Effects: 
## ~ParticipantID (Number of levels: 32) 
##                       Estimate Est.Error l-95% CI u-95% CI Rhat Bulk_ESS
## sd(Intercept)             0.62      0.08     0.48     0.81 1.01      588
## sd(Motion)                0.28      0.04     0.20     0.37 1.00     1357
## cor(Intercept,Motion)    -0.50      0.15    -0.75    -0.16 1.00     1400
##                       Tail_ESS
## sd(Intercept)             1010
## sd(Motion)                1777
## cor(Intercept,Motion)     1873
## 
## Population-Level Effects: 
##                                           Estimate Est.Error l-95% CI u-95% CI
## Intercept                                     2.27      0.12     2.05     2.50
## Stereo                                        0.37      0.05     0.27     0.47
## Motion                                        0.59      0.05     0.49     0.70
## Perspective                                   0.04      0.02    -0.00     0.08
## Shading                                       0.01      0.03    -0.04     0.06
## Dataset                                       0.17      0.05     0.07     0.27
## Dimension                                     0.13      0.05     0.04     0.23
## Stereo:Motion                                 0.38      0.04     0.30     0.46
## Stereo:Perspective                            0.03      0.04    -0.05     0.11
## Motion:Perspective                            0.06      0.04    -0.03     0.14
## Stereo:Shading                                0.09      0.05    -0.01     0.19
## Motion:Shading                                0.15      0.05     0.05     0.24
## Perspective:Shading                          -0.07      0.05    -0.17     0.02
## Stereo:Dataset                               -0.11      0.11    -0.31     0.10
## Motion:Dataset                               -0.01      0.06    -0.13     0.10
## Perspective:Dataset                           0.03      0.04    -0.05     0.10
## Shading:Dataset                               0.07      0.05    -0.03     0.17
## Stereo:Dimension                             -0.06      0.10    -0.26     0.15
## Dataset:Dimension                             0.06      0.10    -0.14     0.25
## Stereo:Motion:Perspective                    -0.03      0.08    -0.19     0.14
## Stereo:Motion:Shading                        -0.06      0.10    -0.25     0.14
## Stereo:Perspective:Shading                    0.08      0.10    -0.12     0.28
## Motion:Perspective:Shading                    0.10      0.10    -0.10     0.29
## Stereo:Motion:Dataset                         0.41      0.11     0.19     0.63
## Stereo:Perspective:Dataset                   -0.05      0.08    -0.21     0.10
## Motion:Perspective:Dataset                   -0.04      0.08    -0.20     0.12
## Stereo:Shading:Dataset                       -0.05      0.10    -0.24     0.15
## Motion:Shading:Dataset                        0.05      0.10    -0.16     0.24
## Perspective:Shading:Dataset                   0.01      0.10    -0.19     0.20
## Stereo:Dataset:Dimension                     -0.03      0.20    -0.43     0.35
## Stereo:Motion:Perspective:Shading             0.19      0.20    -0.20     0.58
## Stereo:Motion:Perspective:Dataset             0.08      0.16    -0.25     0.39
## Stereo:Motion:Shading:Dataset                -0.15      0.20    -0.54     0.23
## Stereo:Perspective:Shading:Dataset           -0.07      0.20    -0.47     0.32
## Motion:Perspective:Shading:Dataset            0.26      0.20    -0.14     0.66
## Stereo:Motion:Perspective:Shading:Dataset     0.26      0.39    -0.52     1.01
## shape_Intercept                               1.17      0.02     1.13     1.22
## shape_Motion                                  0.25      0.05     0.16     0.34
##                                           Rhat Bulk_ESS Tail_ESS
## Intercept                                 1.01      263      732
## Stereo                                    1.00     2246     2642
## Motion                                    1.00      874     1758
## Perspective                               1.00     5648     2768
## Shading                                   1.00     5097     2924
## Dataset                                   1.00     2217     2537
## Dimension                                 1.00     4129     3254
## Stereo:Motion                             1.00     4887     2657
## Stereo:Perspective                        1.00     4527     2604
## Motion:Perspective                        1.00     6393     3332
## Stereo:Shading                            1.00     4232     3001
## Motion:Shading                            1.00     4643     3297
## Perspective:Shading                       1.00     5635     3019
## Stereo:Dataset                            1.00     2025     2354
## Motion:Dataset                            1.00     4251     2775
## Perspective:Dataset                       1.00     5492     3004
## Shading:Dataset                           1.00     4885     3070
## Stereo:Dimension                          1.00     2377     2667
## Dataset:Dimension                         1.00     2492     2890
## Stereo:Motion:Perspective                 1.00     4841     3102
## Stereo:Motion:Shading                     1.00     4777     3204
## Stereo:Perspective:Shading                1.00     5092     2935
## Motion:Perspective:Shading                1.00     6211     3028
## Stereo:Motion:Dataset                     1.00     4573     2764
## Stereo:Perspective:Dataset                1.00     4950     2860
## Motion:Perspective:Dataset                1.00     5356     3000
## Stereo:Shading:Dataset                    1.00     5059     3090
## Motion:Shading:Dataset                    1.00     5083     3290
## Perspective:Shading:Dataset               1.00     4987     3142
## Stereo:Dataset:Dimension                  1.00     2114     2301
## Stereo:Motion:Perspective:Shading         1.00     5230     2580
## Stereo:Motion:Perspective:Dataset         1.00     5440     2786
## Stereo:Motion:Shading:Dataset             1.00     5032     3184
## Stereo:Perspective:Shading:Dataset        1.00     4767     2573
## Motion:Perspective:Shading:Dataset        1.00     4181     3305
## Stereo:Motion:Perspective:Shading:Dataset 1.00     3891     2759
## shape_Intercept                           1.00     5674     2748
## shape_Motion                              1.00     5814     2770
## 
## Samples were drawn using sampling(NUTS). For each parameter, Bulk_ESS
## and Tail_ESS are effective sample size measures, and Rhat is the potential
## scale reduction factor on split chains (at convergence, Rhat = 1).

## 
## Computed from 4000 by 3320 log-likelihood matrix
## 
##          Estimate    SE
## elpd_loo  -9988.5  72.3
## p_loo       114.9   6.6
## looic     19977.0 144.5
## ------
## Monte Carlo SE of elpd_loo is NA.
## 
## Pareto k diagnostic values:
##                          Count Pct.    Min. n_eff
## (-Inf, 0.5]   (good)     3314  99.8%   354       
##  (0.5, 0.7]   (ok)          5   0.2%   118       
##    (0.7, 1]   (bad)         1   0.0%   423       
##    (1, Inf)   (very bad)    0   0.0%   <NA>      
## See help('pareto-k-diagnostic') for details.
## 
## Computed from 4000 by 3320 log-likelihood matrix
## 
##           Estimate    SE
## elpd_waic  -9987.9  72.2
## p_waic       114.3   6.5
## waic       19975.8 144.4
## 
## 40 (1.2%) p_waic estimates greater than 0.4. We recommend trying loo instead.
summary(m_brms_time_gamma_motion)
##  Family: gamma 
##   Links: mu = log; shape = log 
## Formula: ResponseTime | cens(cen) ~ 0 + Intercept + Stereo * Motion * Perspective * Shading * Dataset + Stereo * Dataset * Dimension + (1 + Motion | ParticipantID) 
##          shape ~ 0 + Intercept + Motion
##    Data: trials (Number of observations: 3320) 
## Samples: 2 chains, each with iter = 4000; warmup = 2000; thin = 1;
##          total post-warmup samples = 4000
## 
## Group-Level Effects: 
## ~ParticipantID (Number of levels: 32) 
##                       Estimate Est.Error l-95% CI u-95% CI Rhat Bulk_ESS
## sd(Intercept)             0.62      0.08     0.48     0.81 1.01      588
## sd(Motion)                0.28      0.04     0.20     0.37 1.00     1357
## cor(Intercept,Motion)    -0.50      0.15    -0.75    -0.16 1.00     1400
##                       Tail_ESS
## sd(Intercept)             1010
## sd(Motion)                1777
## cor(Intercept,Motion)     1873
## 
## Population-Level Effects: 
##                                           Estimate Est.Error l-95% CI u-95% CI
## Intercept                                     2.27      0.12     2.05     2.50
## Stereo                                        0.37      0.05     0.27     0.47
## Motion                                        0.59      0.05     0.49     0.70
## Perspective                                   0.04      0.02    -0.00     0.08
## Shading                                       0.01      0.03    -0.04     0.06
## Dataset                                       0.17      0.05     0.07     0.27
## Dimension                                     0.13      0.05     0.04     0.23
## Stereo:Motion                                 0.38      0.04     0.30     0.46
## Stereo:Perspective                            0.03      0.04    -0.05     0.11
## Motion:Perspective                            0.06      0.04    -0.03     0.14
## Stereo:Shading                                0.09      0.05    -0.01     0.19
## Motion:Shading                                0.15      0.05     0.05     0.24
## Perspective:Shading                          -0.07      0.05    -0.17     0.02
## Stereo:Dataset                               -0.11      0.11    -0.31     0.10
## Motion:Dataset                               -0.01      0.06    -0.13     0.10
## Perspective:Dataset                           0.03      0.04    -0.05     0.10
## Shading:Dataset                               0.07      0.05    -0.03     0.17
## Stereo:Dimension                             -0.06      0.10    -0.26     0.15
## Dataset:Dimension                             0.06      0.10    -0.14     0.25
## Stereo:Motion:Perspective                    -0.03      0.08    -0.19     0.14
## Stereo:Motion:Shading                        -0.06      0.10    -0.25     0.14
## Stereo:Perspective:Shading                    0.08      0.10    -0.12     0.28
## Motion:Perspective:Shading                    0.10      0.10    -0.10     0.29
## Stereo:Motion:Dataset                         0.41      0.11     0.19     0.63
## Stereo:Perspective:Dataset                   -0.05      0.08    -0.21     0.10
## Motion:Perspective:Dataset                   -0.04      0.08    -0.20     0.12
## Stereo:Shading:Dataset                       -0.05      0.10    -0.24     0.15
## Motion:Shading:Dataset                        0.05      0.10    -0.16     0.24
## Perspective:Shading:Dataset                   0.01      0.10    -0.19     0.20
## Stereo:Dataset:Dimension                     -0.03      0.20    -0.43     0.35
## Stereo:Motion:Perspective:Shading             0.19      0.20    -0.20     0.58
## Stereo:Motion:Perspective:Dataset             0.08      0.16    -0.25     0.39
## Stereo:Motion:Shading:Dataset                -0.15      0.20    -0.54     0.23
## Stereo:Perspective:Shading:Dataset           -0.07      0.20    -0.47     0.32
## Motion:Perspective:Shading:Dataset            0.26      0.20    -0.14     0.66
## Stereo:Motion:Perspective:Shading:Dataset     0.26      0.39    -0.52     1.01
## shape_Intercept                               1.17      0.02     1.13     1.22
## shape_Motion                                  0.25      0.05     0.16     0.34
##                                           Rhat Bulk_ESS Tail_ESS
## Intercept                                 1.01      263      732
## Stereo                                    1.00     2246     2642
## Motion                                    1.00      874     1758
## Perspective                               1.00     5648     2768
## Shading                                   1.00     5097     2924
## Dataset                                   1.00     2217     2537
## Dimension                                 1.00     4129     3254
## Stereo:Motion                             1.00     4887     2657
## Stereo:Perspective                        1.00     4527     2604
## Motion:Perspective                        1.00     6393     3332
## Stereo:Shading                            1.00     4232     3001
## Motion:Shading                            1.00     4643     3297
## Perspective:Shading                       1.00     5635     3019
## Stereo:Dataset                            1.00     2025     2354
## Motion:Dataset                            1.00     4251     2775
## Perspective:Dataset                       1.00     5492     3004
## Shading:Dataset                           1.00     4885     3070
## Stereo:Dimension                          1.00     2377     2667
## Dataset:Dimension                         1.00     2492     2890
## Stereo:Motion:Perspective                 1.00     4841     3102
## Stereo:Motion:Shading                     1.00     4777     3204
## Stereo:Perspective:Shading                1.00     5092     2935
## Motion:Perspective:Shading                1.00     6211     3028
## Stereo:Motion:Dataset                     1.00     4573     2764
## Stereo:Perspective:Dataset                1.00     4950     2860
## Motion:Perspective:Dataset                1.00     5356     3000
## Stereo:Shading:Dataset                    1.00     5059     3090
## Motion:Shading:Dataset                    1.00     5083     3290
## Perspective:Shading:Dataset               1.00     4987     3142
## Stereo:Dataset:Dimension                  1.00     2114     2301
## Stereo:Motion:Perspective:Shading         1.00     5230     2580
## Stereo:Motion:Perspective:Dataset         1.00     5440     2786
## Stereo:Motion:Shading:Dataset             1.00     5032     3184
## Stereo:Perspective:Shading:Dataset        1.00     4767     2573
## Motion:Perspective:Shading:Dataset        1.00     4181     3305
## Stereo:Motion:Perspective:Shading:Dataset 1.00     3891     2759
## shape_Intercept                           1.00     5674     2748
## shape_Motion                              1.00     5814     2770
## 
## Samples were drawn using sampling(NUTS). For each parameter, Bulk_ESS
## and Tail_ESS are effective sample size measures, and Rhat is the potential
## scale reduction factor on split chains (at convergence, Rhat = 1).
if(REBUILD_FLAG){
  
  trials <- trials %>%
      mutate(cen = ifelse(ResponseTime > 60, "left", "none"))

  m_time_gamma_motion_simple <- bf(ResponseTime | cens(cen) ~  
                      0 + Intercept +
                      Stereo*Motion*Perspective*Shading*Dataset +  
                      Stereo*Dataset*Dimension + 
                      (1 + Motion | ParticipantID),
                      family = Gamma(link="log")
                     ) 
    
  priors_time_gamma_motion_simple <- c(
                  prior(normal(0, 2.5), class = "b"),
                  prior(normal(0, 30), class = "b", coef = "Intercept"), 
                  prior(gamma(1, 20), class = "shape"),
                  prior(student_t(3, 0, 10), class = "sd")
                ) 
    
  m_brms_time_gamma_motion_simple <<- brm(m_time_gamma_motion, 
                              data = trials, family = Gamma(link="log"),  prior = priors_time_gamma_motion, 
                              chains = N_CHAINS, iter = N_ITER, warmup = N_WARMUP, thin = N_THIN, cores = N_CORES,
                              control = list(adapt_delta = DELTA, max_treedepth = TREE_DEPTH))
  
  saveRDS(file = "rds/m_brms_time_gamma_motion_simple.rds", m_brms_time_gamma_motion_simple)
  
}else{
  m_brms_time_gamma_motion_simple <<- readRDS("rds/m_brms_time_gamma_motion_simple.rds")
}

model_diagnostics(m_brms_time_gamma_motion_simple)
## [1] "model diagnostics"
##  Family: gamma 
##   Links: mu = log; shape = log 
## Formula: ResponseTime | cens(cen) ~ 0 + Intercept + Stereo * Motion * Perspective * Shading * Dataset + Stereo * Dataset * Dimension + (1 + Motion | ParticipantID) 
##          shape ~ 0 + Intercept + Motion
##    Data: trials (Number of observations: 3320) 
## Samples: 2 chains, each with iter = 4000; warmup = 2000; thin = 1;
##          total post-warmup samples = 4000
## 
## Group-Level Effects: 
## ~ParticipantID (Number of levels: 32) 
##                       Estimate Est.Error l-95% CI u-95% CI Rhat Bulk_ESS
## sd(Intercept)             0.62      0.08     0.49     0.80 1.00      628
## sd(Motion)                0.28      0.04     0.20     0.38 1.00     1286
## cor(Intercept,Motion)    -0.51      0.15    -0.76    -0.19 1.00     1415
##                       Tail_ESS
## sd(Intercept)              966
## sd(Motion)                1657
## cor(Intercept,Motion)     2004
## 
## Population-Level Effects: 
##                                           Estimate Est.Error l-95% CI u-95% CI
## Intercept                                     2.28      0.12     2.05     2.52
## Stereo                                        0.37      0.05     0.27     0.47
## Motion                                        0.59      0.05     0.49     0.69
## Perspective                                   0.04      0.02     0.00     0.08
## Shading                                       0.01      0.02    -0.04     0.06
## Dataset                                       0.16      0.05     0.06     0.27
## Dimension                                     0.13      0.05     0.03     0.23
## Stereo:Motion                                 0.38      0.04     0.30     0.46
## Stereo:Perspective                            0.03      0.04    -0.05     0.11
## Motion:Perspective                            0.06      0.04    -0.03     0.14
## Stereo:Shading                                0.09      0.05    -0.01     0.18
## Motion:Shading                                0.14      0.05     0.04     0.24
## Perspective:Shading                          -0.07      0.05    -0.17     0.02
## Stereo:Dataset                               -0.10      0.11    -0.31     0.10
## Motion:Dataset                               -0.01      0.06    -0.13     0.10
## Perspective:Dataset                           0.03      0.04    -0.05     0.10
## Shading:Dataset                               0.07      0.05    -0.03     0.16
## Stereo:Dimension                             -0.06      0.10    -0.26     0.15
## Dataset:Dimension                             0.06      0.10    -0.13     0.26
## Stereo:Motion:Perspective                    -0.02      0.08    -0.18     0.14
## Stereo:Motion:Shading                        -0.06      0.10    -0.26     0.14
## Stereo:Perspective:Shading                    0.08      0.10    -0.11     0.27
## Motion:Perspective:Shading                    0.10      0.10    -0.10     0.30
## Stereo:Motion:Dataset                         0.41      0.12     0.18     0.64
## Stereo:Perspective:Dataset                   -0.05      0.08    -0.22     0.11
## Motion:Perspective:Dataset                   -0.04      0.08    -0.20     0.13
## Stereo:Shading:Dataset                       -0.05      0.10    -0.25     0.16
## Motion:Shading:Dataset                        0.05      0.10    -0.14     0.24
## Perspective:Shading:Dataset                   0.01      0.10    -0.18     0.19
## Stereo:Dataset:Dimension                     -0.04      0.20    -0.43     0.35
## Stereo:Motion:Perspective:Shading             0.19      0.20    -0.20     0.56
## Stereo:Motion:Perspective:Dataset             0.08      0.17    -0.24     0.39
## Stereo:Motion:Shading:Dataset                -0.16      0.20    -0.55     0.23
## Stereo:Perspective:Shading:Dataset           -0.08      0.19    -0.45     0.31
## Motion:Perspective:Shading:Dataset            0.27      0.20    -0.14     0.66
## Stereo:Motion:Perspective:Shading:Dataset     0.26      0.40    -0.50     1.06
## shape_Intercept                               1.17      0.02     1.13     1.22
## shape_Motion                                  0.25      0.05     0.16     0.35
##                                           Rhat Bulk_ESS Tail_ESS
## Intercept                                 1.00      349      527
## Stereo                                    1.00     2556     2653
## Motion                                    1.00      758     1394
## Perspective                               1.00     5636     2943
## Shading                                   1.00     5526     3058
## Dataset                                   1.00     2610     2689
## Dimension                                 1.00     3955     3063
## Stereo:Motion                             1.00     5317     3104
## Stereo:Perspective                        1.00     4829     3003
## Motion:Perspective                        1.00     5023     2508
## Stereo:Shading                            1.00     5417     2898
## Motion:Shading                            1.00     5682     2907
## Perspective:Shading                       1.00     5256     3065
## Stereo:Dataset                            1.00     2489     2975
## Motion:Dataset                            1.00     5709     3185
## Perspective:Dataset                       1.00     6050     3137
## Shading:Dataset                           1.00     4946     3151
## Stereo:Dimension                          1.00     2669     2968
## Dataset:Dimension                         1.00     2770     2857
## Stereo:Motion:Perspective                 1.00     4367     2885
## Stereo:Motion:Shading                     1.00     5435     3152
## Stereo:Perspective:Shading                1.00     4465     2914
## Motion:Perspective:Shading                1.00     5049     3176
## Stereo:Motion:Dataset                     1.00     5008     2769
## Stereo:Perspective:Dataset                1.00     5056     3123
## Motion:Perspective:Dataset                1.00     5304     3045
## Stereo:Shading:Dataset                    1.00     5379     2716
## Motion:Shading:Dataset                    1.00     4333     2781
## Perspective:Shading:Dataset               1.00     3984     3101
## Stereo:Dataset:Dimension                  1.00     2661     2788
## Stereo:Motion:Perspective:Shading         1.00     4646     3093
## Stereo:Motion:Perspective:Dataset         1.00     5388     2870
## Stereo:Motion:Shading:Dataset             1.00     4071     2912
## Stereo:Perspective:Shading:Dataset        1.00     5116     2702
## Motion:Perspective:Shading:Dataset        1.00     5631     3050
## Stereo:Motion:Perspective:Shading:Dataset 1.00     5647     3238
## shape_Intercept                           1.00     5217     3033
## shape_Motion                              1.00     5531     2850
## 
## Samples were drawn using sampling(NUTS). For each parameter, Bulk_ESS
## and Tail_ESS are effective sample size measures, and Rhat is the potential
## scale reduction factor on split chains (at convergence, Rhat = 1).

## 
## Computed from 4000 by 3320 log-likelihood matrix
## 
##          Estimate    SE
## elpd_loo  -9988.0  72.2
## p_loo       114.5   6.5
## looic     19976.1 144.5
## ------
## Monte Carlo SE of elpd_loo is NA.
## 
## Pareto k diagnostic values:
##                          Count Pct.    Min. n_eff
## (-Inf, 0.5]   (good)     3311  99.7%   553       
##  (0.5, 0.7]   (ok)          8   0.2%   227       
##    (0.7, 1]   (bad)         1   0.0%   86        
##    (1, Inf)   (very bad)    0   0.0%   <NA>      
## See help('pareto-k-diagnostic') for details.
## 
## Computed from 4000 by 3320 log-likelihood matrix
## 
##           Estimate    SE
## elpd_waic  -9987.4  72.2
## p_waic       113.9   6.5
## waic       19974.8 144.4
## 
## 40 (1.2%) p_waic estimates greater than 0.4. We recommend trying loo instead.
draw_coefs <- function(m, 
                       target_coefs, 
                       title = "", 
                       fill_color = "white" , 
                       breaks_x = seq(0, 1, by = 0.5)
                       ){
  post_samples <- posterior_samples(m, pars = "b_")
  
  b_CIs <-  post_samples %>% 
            gather(key = "coefs", value = "value") %>% 
            group_by(coefs) %>%
            median_qi() %>%
            ungroup() %>%
            mutate(lower95 = exp(.lower), upper95 = exp(.upper), median = exp(value))%>%
            select(coefs, median, lower95, upper95) %>%
            mutate(coefs = factor(coefs, levels = rev(target_coefs)), 
                 s_lower95 = round(lower95, 3), s_upper95 = round(upper95, 3),
                 CIs = paste0(round(median, 3), ' [', s_lower95, ',',  s_upper95, ']'))
    
  b_samples <-  post_samples %>% 
    gather(key = "coefs", value = "value") %>% 
    ungroup() 
    
  g <- b_samples %>%
    filter(coefs %in% target_coefs) %>%
    mutate(coefs = factor(coefs, levels = rev(target_coefs)), value = exp(value)) %>%
    ggplot(aes(x = value, y = coefs)) + 
      stat_slab(fill = fill_color, n = (N_ITER - N_WARMUP) * N_CHAINS) + 
      geom_richtext(data = b_CIs, mapping = aes(x = tail(breaks_x, n = 1) * 1.1, y = coefs, label = CIs), 
                 size = 2, 
                 fill = NA, label.color = NA, # remove background and outline
                 label.padding = grid::unit(rep(0, 4), "pt")) + 
      geom_vline(xintercept = 1, linetype = "dashed", size = LINE_SIZE) +
      scale_x_continuous(breaks = breaks_x, expand = expansion(mult = c(0.01, .5))) +
      coord_cartesian(xlim = c(breaks_x[1], tail(breaks_x, n = 1))) +
      ggtitle(title) + xlab("exp(beta)") + ylab("") 
  
  return(g)
}

How do the six manipulated variables affect participants’ performance?

target_coefs_rq1 <- c("b_Stereo", "b_Motion", "b_Perspective", "b_Shading", "b_Dimension", "b_Dataset")


g1_rq1 <- draw_coefs(m_brms_error_hurdle_gamma, 
                 target_coefs_rq1, 
                 "Error magnitude (%)", 
                 ERROR_COLOR, 
                 breaks_x = seq(0.75, 1.5, by = 0.25))
g2_rq1 <- draw_coefs(m_brms_time_gamma_motion, 
                 target_coefs_rq1, 
                 "Response time (s)",
                 TIME_COLOR, 
                 breaks_x = seq(0.85, 2.2, by = 0.45))

g_combined_rq1 <- ggarrange(g1_rq1, g2_rq1, ncol = 2)

ggsave(g_combined_rq1, filename = "pic/RQ1-mean-effects.pdf", width = 8, height = 2.5, units = "in")

g_combined_rq1

Interaction effects

target_coefs_rq2 <- c("b_Stereo:Motion", 
                  "b_Stereo:Perspective", 
                  "b_Stereo:Shading", 
                  "b_Stereo:Dimension", 
                  "b_Stereo:Dataset", 
                  "b_Motion:Perspective", 
                  "b_Motion:Shading", 
                  "b_Motion:Dataset", 
                  "b_Perspective:Shading", 
                  "b_Perspective:Dataset", 
                  "b_Shading:Dataset", 
                  "b_Dataset:Dimension")


g1_rq2 <- draw_coefs(m_brms_error_hurdle_gamma, 
                 target_coefs_rq2, 
                 "Error magnitude (%)", 
                 ERROR_COLOR, 
                 breaks_x = seq(0.5, 2.1, by = 0.5))
g2_rq2 <- draw_coefs(m_brms_time_gamma_motion, 
                 target_coefs_rq2, 
                 "Response time (s)",
                 TIME_COLOR, 
                 breaks_x = seq(0.5, 2.1, by = 0.5))

g_combined_rq2 <- ggarrange(g1_rq2, g2_rq2, ncol = 2)

ggsave(g_combined_rq2, filename = "pic/RQ2-interaction-effects.pdf", width = 8, height = 3.5, units = "in")

g_combined_rq2

draw_post <- function( m, 
                       title = "", 
                       fill_color = "white" , 
                       palette_in, 
                       adjust_in,
                       breaks_x = seq(0, 1, by = 0.5)
                       ){
  
  post_samples <- trials %>%
    select(Stereo, Motion, Perspective, Shading, Dimension, Dataset) %>%
    distinct() %>%
    add_fitted_draws(m, value = "sample_value", scale = "linear", re_formula = NA, allow_new_levels = FALSE) %>%
    select(-c(".chain", ".iteration"))%>%
    ungroup()%>%
    select(Stereo, Motion, Perspective, Shading, Dimension, Dataset, sample_value) %>%
    gather(key = "levels_names", value = "levels_values", Stereo, Motion, Perspective, Shading, Dimension, Dataset)  %>%
    mutate(levels_values = factor(levels_values), sample_value = exp(sample_value))
  
  b_CIs <-  post_samples %>%
            group_by(levels_names, levels_values) %>%
            median_qi()  %>%
            ungroup() %>%
            mutate(lower95 = .lower, upper95 = .upper, median = sample_value)%>%
            mutate(levels_values = factor(levels_values), 
                 s_lower95 = round(lower95, 3), s_upper95 = round(upper95, 3),
                 CIs = paste0(round(median, 3), ' [', s_lower95, ',',  s_upper95, ']'))
  
  g <- post_samples %>%
      ggplot(aes(x = sample_value, y = levels_names)) +
      geom_density_ridges(
                aes(fill = levels_values, color = levels_values), 
                rel_min_height = 0.001,
                alpha = 0.5,
                scale = 0.8,
                size = 0.5
                ) +
      geom_richtext(data = b_CIs, mapping = aes(x = tail(breaks_x, n = 1) * 1.2, 
                                                y = levels_names, 
                                                color = levels_values, 
                                                label = CIs), 
                 label.color = NA,
                 size = 2, 
                 fill = NA, 
                 label.padding = grid::unit(rep(0, 4), "pt")) + 
      scale_x_continuous(breaks = breaks_x, expand = expansion(mult = c(0.01, .5))) +
      scale_fill_manual(values = palette_in) + 
      scale_color_manual(values = palette_in) + 
      coord_cartesian(xlim = c(breaks_x[1], tail(breaks_x, n = 1))) +
      ggtitle(title) + xlab("") + ylab("") 
  
  return(g)
}

g1_rq3 <- draw_post(m_brms_error_hurdle_gamma, 
                 "Error magnitude (%)", 
                 ERROR_COLOR, 
                 PALETTE_ERROR,
                 adjust_in = 1, 
                 breaks_x = seq(0, 15, by = 5))

g2_rq3 <- draw_post(m_brms_time_gamma_motion, 
                 "Response time (s)",
                 TIME_COLOR, 
                 PALETTE_TIME,
                 adjust_in = 2, 
                 breaks_x = seq(0, 45, by = 15))

g_combined_rq3 <- ggarrange(g1_rq3, g2_rq3, ncol = 2)

ggsave(g_combined_rq3, filename = "pic/RQ3-div.pdf", width = 7, height = 3.5, units = "in")

g_combined_rq3

 global_order <- NULL

 draw_all <- function( m, 
                        title = "", 
                        fill_color = "white" , 
                        palette_in, 
                        adjust_in,
                        breaks_x = seq(0, 1, by = 0.5),
                        which = "error"
                        ){
  
  post_samples <- trials %>%
    select(Stereo, Motion, Perspective, Shading, Dimension, Dataset) %>%
    distinct() %>%
    add_fitted_draws(m, value = "sample_value", scale = "linear", re_formula = NA, allow_new_levels = FALSE) %>%
    select(-c(".chain", ".iteration"))%>%
    ungroup()%>%
    select(Stereo, Motion, Perspective, Shading, Dimension, Dataset, sample_value) %>%
    mutate(  Stereo = recode(Stereo,  `0.5` = "HMD", `-0.5` = "Desktop"),
             Motion = recode(Motion,  `0.5` = "Motion", `-0.5` = "No Motion"),
             Perspective = recode(Perspective, `0.5` = "Perspective", `-0.5` = "Orthographic"),
             Shading = recode(Shading, `0` = "Simple Shading", `0.5` = "Ambient Occlusion", `-0.5` = "Flat Shading"),
             Dataset = recode(Dataset, `0.5` = "CIFAR-10", `-0.5` =  "bAbI"),
             Dimension = recode(Dimension, `-0.5` = "2D t-SNE",  `0.5` = "3D t-SNE"), 
             sample_value = exp(sample_value),
             Condition = paste(Stereo, Motion, Perspective, Shading, Dimension, Dataset, sep = seper))
  
  b_CIs <-  post_samples %>%
            select(Condition, sample_value) %>%
            group_by(Condition) %>%
            median_qi()  %>%
            ungroup() %>%
            mutate(lower95 = .lower, upper95 = .upper, median = sample_value, 
                   s_lower95 = round(lower95, 3), s_upper95 = round(upper95, 3),
                   CIs = paste0(round(median, 3), ' [', s_lower95, ',',  s_upper95, ']')) 
  if(which == "error"){
    b_CIs <- b_CIs %>%
            mutate(Condition = fct_reorder(Condition, .x = median, .desc = TRUE))
    global_order <<- levels(b_CIs$Condition)
  }else{
    b_CIs <- b_CIs %>%
            mutate(Condition = factor(Condition, levels = global_order))
  }
  
  g <- post_samples %>%
      ggplot(aes(x = sample_value, y = Condition)) +
      theme(axis.text.y = element_text(size = 3)) + 
      geom_richtext(data = b_CIs, mapping = aes(x = tail(breaks_x, n = 1) * 1.2, 
                                                y = Condition, 
                                                label = CIs), 
                 label.color = NA,
                 size = 3, 
                 fill = NA, 
                 label.padding = grid::unit(rep(0, 4), "pt")) + 
      geom_density_ridges(
                  fill = fill_color, 
                  rel_min_height = 0.01,
                  alpha = 0.5,
                  scale = 1.5,
                  size = 0.5
                  ) +
      scale_x_continuous(breaks = breaks_x, expand = expansion(mult = c(0.01, .5))) +
      coord_cartesian(xlim = c(breaks_x[1], tail(breaks_x, n = 1))) +
      ggtitle(title) + xlab("") + ylab("") 
  g
  return(g)
}

g1_rq4 <- draw_all(m_brms_error_hurdle_gamma, 
                 "Error magnitude (%)", 
                 ERROR_COLOR, 
                 PALETTE_ERROR,
                 adjust_in = 1, 
                 breaks_x = seq(3, 15, by = 4),
                 which = "error")

g2_rq4 <- draw_all(m_brms_time_gamma_motion, 
                 "Response time (s)",
                 TIME_COLOR, 
                 PALETTE_TIME,
                 adjust_in = 1, 
                 breaks_x = seq(0, 36, by = 12),
                 which = "time")

g_combined_rq4 <- ggarrange(g1_rq4, g2_rq4, ncol = 2)

ggsave(g_combined_rq4, filename = "pic/RQ4-all.pdf", width = 15, height = 12, units = "in")

g_combined_rq4

# Extra figures

demographics <- metas %>% select(familiarML, familiarDR, familiarVIS, familiarVR)
demographics_stack <- stack(demographics)
demographics_stack$overall <- paste(demographics_stack$values, demographics_stack$ind, sep = " ")

demographics_table <- aggregate(data = demographics_stack, values ~ overall, length)
demographics_table$item <- sapply(demographics_table$overall, function(x){return(strsplit(x, split = " ")[[1]][2])})  
demographics_table$option <- sapply(demographics_table$overall, function(x){return(strsplit(x, split = " ")[[1]][1])})  

g_demographics <- ggplot() %>%
  + geom_tile(data = demographics_table, aes(x = option, y = item, fill = values)) %>%
  + scale_fill_continuous(low = "#ffffff", high = ERROR_COLOR, limits = c(0,24), breaks = c(0, 8, 16, 24)) + xlab("scale") + ylab("")
g_demographics

ggsave(file = "pic/demographics.pdf", g_demographics, width = 2.7, height = 2, units = "in")

Accuracy range examples

acc1 <- c(0.3978, 0.4054, 0.5998, 0.602, 0.605, 0.6052, 0.6528, 0.653, 0.6614, 0.6762, 0.6956, 0.7044, 0.713, 0.7162, 0.725, 0.73, 0.7326, 0.7362, 0.7398, 0.745, 0.7458, 0.7476, 0.751, 0.7538, 0.7668, 0.767, 0.7686, 0.7686, 0.769, 0.7744, 0.7748, 0.7754, 0.7758, 0.7778, 0.7788, 0.7794, 0.7806, 0.7828, 0.7834, 0.7848, 0.7858, 0.7882, 0.7888, 0.789, 0.7894, 0.7914, 0.7932, 0.7946, 0.7948, 0.7952, 0.7954, 0.7966, 0.7968, 0.7976, 0.799, 0.7992, 0.801, 0.8014, 0.8018, 0.8026, 0.8028, 0.8032, 0.804, 0.804, 0.8046, 0.805, 0.8056, 0.8058, 0.806, 0.8066, 0.8068, 0.8072, 0.8076, 0.808, 0.8084, 0.8098, 0.8108, 0.8108, 0.8108, 0.8118, 0.812, 0.8132, 0.8138, 0.8138, 0.8148, 0.816, 0.816, 0.8172, 0.8176, 0.8182, 0.8192, 0.8202, 0.8204, 0.8204, 0.8206, 0.8206, 0.8206, 0.8212, 0.8212, 0.822, 0.8224, 0.824, 0.8244, 0.8254, 0.8258, 0.8268, 0.8274, 0.8276, 0.8286, 0.8286, 0.829, 0.8302, 0.8314, 0.8314, 0.8326, 0.833, 0.8332, 0.8338, 0.834, 0.8352, 0.8358, 0.8364, 0.837, 0.8384, 0.8386, 0.8386, 0.8396, 0.8406, 0.8408, 0.8408, 0.8414, 0.842, 0.8428, 0.8436, 0.8442, 0.8444, 0.8444, 0.8446, 0.845, 0.8452, 0.847, 0.8472, 0.8472, 0.8474, 0.8474, 0.848, 0.85, 0.8504, 0.8506, 0.8514, 0.8516, 0.8518, 0.8534, 0.855, 0.8578, 0.858, 0.8582, 0.8584, 0.8584, 0.861, 0.8634, 0.8662, 0.8912, 0.8914, 0.8922, 0.8942, 0.8954, 0.896, 0.896, 0.896, 0.8962, 0.8964, 0.8966, 0.8968, 0.8972, 0.8974, 0.8978, 0.8978, 0.8978, 0.8982, 0.8982, 0.8984, 0.8984, 0.8986, 0.899, 0.8992, 0.8992, 0.8994, 0.8996, 0.8998, 0.8998, 0.8998, 0.8998, 0.8998, 0.9, 0.9, 0.9002, 0.9002, 0.9002, 0.9002, 0.9002, 0.9004, 0.9006, 0.9006, 0.9008, 0.901, 0.901, 0.9012, 0.9012, 0.9014, 0.9014, 0.9014, 0.9016, 0.9018, 0.9018, 0.9018, 0.902, 0.902, 0.902, 0.9022, 0.9024, 0.9026, 0.9026, 0.9026, 0.9026, 0.9026, 0.9028, 0.9028, 0.903, 0.903, 0.903, 0.903, 0.903, 0.9032, 0.9032, 0.9036, 0.9036, 0.9036, 0.9036, 0.9036, 0.9038, 0.9038, 0.9038, 0.9038, 0.9044, 0.9044, 0.9044, 0.9044, 0.9046, 0.9046, 0.905, 0.9052, 0.9052, 0.9052, 0.9054, 0.9054, 0.9056, 0.9056, 0.9056, 0.9058, 0.9058, 0.9058, 0.9058, 0.906, 0.906, 0.906, 0.906, 0.906, 0.9062, 0.9064, 0.9064, 0.9066, 0.9066, 0.9068, 0.9068, 0.9068, 0.9068, 0.907, 0.907, 0.9074, 0.9074, 0.9074, 0.908, 0.908, 0.9086, 0.9092, 0.9092, 0.9092, 0.9092, 0.9094, 0.9094, 0.9098, 0.9098, 0.91, 0.9106, 0.9114, 0.9114, 0.9126, 0.913, 0.9146)


acc2 <- c(0.192, 0.246, 0.394, 0.405, 0.427, 0.439, 0.476, 0.485, 0.488, 0.488, 0.49, 0.491, 0.491, 0.492, 0.492, 0.494, 0.496, 0.496, 0.497, 0.497, 0.497, 0.498, 0.498, 0.499, 0.499, 0.499, 0.5, 0.5, 0.5, 0.5, 0.5, 0.501, 0.504, 0.505, 0.505, 0.506, 0.506, 0.507, 0.507, 0.508, 0.508, 0.508, 0.508, 0.509, 0.509, 0.51, 0.51, 0.511, 0.511, 0.512, 0.512, 0.514, 0.515, 0.515, 0.515, 0.516, 0.522, 0.526, 0.53, 0.531, 0.531, 0.531, 0.534, 0.542, 0.548, 0.559, 0.564, 0.6, 0.625, 0.658, 0.681, 0.697, 0.722, 0.724, 0.726, 0.731, 0.735, 0.739, 0.739, 0.739, 0.739, 0.743, 0.747, 0.748, 0.753, 0.762, 0.764, 0.767, 0.776, 0.79, 0.79, 0.792, 0.795, 0.802, 0.803, 0.81, 0.813, 0.813, 0.817, 0.819, 0.82, 0.82, 0.823, 0.824, 0.824, 0.826, 0.828, 0.828, 0.834, 0.837, 0.838, 0.839, 0.841, 0.841, 0.842, 0.842, 0.843, 0.843, 0.844, 0.847, 0.848, 0.852, 0.853, 0.854, 0.854, 0.854, 0.856, 0.857, 0.857, 0.857, 0.859, 0.86, 0.862, 0.863, 0.863, 0.864, 0.865, 0.866, 0.871, 0.871, 0.874, 0.874, 0.875, 0.876, 0.879, 0.88, 0.881, 0.883, 0.883, 0.883, 0.884, 0.886, 0.886, 0.886, 0.887, 0.887, 0.888, 0.891, 0.891, 0.893, 0.893, 0.894, 0.896, 0.896, 0.896, 0.897, 0.898, 0.899, 0.901, 0.901, 0.901, 0.901, 0.902, 0.903, 0.904, 0.904, 0.904, 0.905, 0.905, 0.905, 0.907, 0.908, 0.908, 0.909, 0.91, 0.91, 0.91, 0.911, 0.911, 0.911, 0.911, 0.912, 0.912, 0.913, 0.913, 0.914, 0.917, 0.917, 0.92, 0.922)

acc3 <- c(0.838, 0.839, 0.8405, 0.8505, 0.8575, 0.8575, 0.865, 0.872, 0.872, 0.873, 0.8735, 0.8745, 0.882, 0.882, 0.8835, 0.885, 0.8865, 0.887, 0.887, 0.892, 0.892, 0.8935, 0.895, 0.8955, 0.896, 0.897, 0.8975, 0.898, 0.898, 0.8985, 0.8985, 0.899, 0.8995, 0.8995, 0.8995, 0.9, 0.9015, 0.902, 0.9025, 0.9035, 0.904, 0.9045, 0.9045, 0.905, 0.9055, 0.9055, 0.906, 0.906, 0.9075, 0.908, 0.9085, 0.9085, 0.9085, 0.9085, 0.9095, 0.9095, 0.9105, 0.9105, 0.9105, 0.911, 0.9115, 0.9115, 0.9115, 0.9115, 0.9125, 0.9125, 0.9125, 0.9125, 0.9125, 0.9125, 0.913, 0.9135, 0.9135, 0.9135, 0.914, 0.914, 0.914, 0.9145, 0.915, 0.915, 0.915, 0.9155, 0.9155, 0.9155, 0.9155, 0.916, 0.916, 0.916, 0.9165, 0.9165, 0.917, 0.917, 0.917, 0.917, 0.9175, 0.918, 0.918, 0.918, 0.918, 0.918, 0.918, 0.918, 0.9185, 0.9185, 0.9185, 0.9185, 0.9185, 0.919, 0.919, 0.919, 0.9195, 0.9195, 0.9195, 0.9195, 0.9195, 0.9195, 0.9195, 0.9195, 0.9195, 0.9195, 0.92, 0.92, 0.92, 0.92, 0.9205, 0.9205, 0.9205, 0.9205, 0.921, 0.921, 0.9215, 0.9215, 0.9215, 0.9215, 0.9215, 0.922, 0.922, 0.922, 0.922, 0.922, 0.922, 0.9225, 0.9225, 0.9225, 0.9225, 0.923, 0.923, 0.923, 0.923, 0.9235, 0.9235, 0.9235, 0.9235, 0.9235, 0.9235, 0.924, 0.924, 0.924, 0.924, 0.9245, 0.9245, 0.9245, 0.9245, 0.9245, 0.9245, 0.9245, 0.9245, 0.925, 0.925, 0.925, 0.925, 0.9255, 0.9255, 0.9255, 0.9255, 0.9255, 0.9255, 0.9255, 0.9255, 0.926, 0.926, 0.926, 0.926, 0.926, 0.9265, 0.9265, 0.927, 0.927, 0.927, 0.927, 0.927, 0.927, 0.927, 0.927, 0.927, 0.9275, 0.9275, 0.9275, 0.9275, 0.928, 0.928, 0.928, 0.928, 0.928, 0.928, 0.928, 0.928, 0.9285, 0.9285, 0.9285, 0.9285, 0.9285, 0.9285, 0.929, 0.929, 0.929, 0.929, 0.929, 0.929, 0.9295, 0.9295, 0.93, 0.93, 0.93, 0.93, 0.93, 0.9305, 0.9305, 0.9305, 0.931, 0.9315, 0.9315, 0.9315, 0.9315, 0.932, 0.9325, 0.9325, 0.933, 0.9335, 0.9335, 0.9335, 0.934, 0.9345, 0.9345, 0.936, 0.936, 0.937, 0.937, 0.94, 0.9405)


g_bar1 <- ggplot() %>%
         + geom_linerange(aes(ymax = 1, ymin = 0, x = acc1), alpha = 0.2, color = ERROR_COLOR) %>%
         + scale_x_continuous(limits=c(0,1), breaks = seq(0, 1, by = 0.1), expand=c(0,0))%>%
         + scale_y_continuous(limits=c(0,1), breaks = seq(0, 1, by = 1), expand=c(0,0))

g_bar2 <- ggplot() %>%
  + geom_linerange(aes(ymax = 1, ymin = 0, x = acc2), alpha = 0.2, color = ERROR_COLOR)%>%
  + scale_x_continuous(limits=c(0,1), breaks = seq(0, 1, by = 0.1), expand=c(0,0))%>%
  + scale_y_continuous(limits=c(0,1), breaks = seq(0, 1, by = 1), expand=c(0,0))

g_bar3 <- ggplot() %>%
  + geom_linerange(aes(ymax = 1, ymin = 0, x = acc3), alpha = 0.2, color = ERROR_COLOR)%>%
  + scale_x_continuous(limits=c(0,1), breaks = seq(0, 1, by = 0.1), expand=c(0,0))%>%
  + scale_y_continuous(limits=c(0,1), breaks = seq(0, 1, by = 1), expand=c(0,0))

g_bars <- ggarrange(g_bar1, g_bar2, g_bar3, ncol = 1, align="v")

ggsave(file = "pic/acc_bar.pdf", g_bars , width = 4, height = 1.6, units = "in")